Smart Diff (macos-10.15): Failure

Expected tolerance 'same' but got 'mixed-worse' instead

===================================================================
--- .github/macos-10.15-expected.txt
+++ results-tests.txt
@@ -15,42 +15,50 @@
memcheck/tests/client-msg (stderr)
memcheck/tests/clientperm (stderr)
memcheck/tests/cxx17_aligned_new (stderr)
memcheck/tests/darwin/deep_badparam (stdout)
+memcheck/tests/darwin/env (stderr)
memcheck/tests/darwin/pth-supp (stderr)
memcheck/tests/darwin/scalar (stderr)
memcheck/tests/darwin/scalar_nocancel (stderr)
memcheck/tests/deep-backtrace (stderr)
memcheck/tests/demangle (stderr)
memcheck/tests/descr_belowsp (stderr)
-memcheck/tests/fprw (stderr)
memcheck/tests/gone_abrt_xml (stderr)
memcheck/tests/inlinfo (stderr)
memcheck/tests/inlinfosupp (stderr)
memcheck/tests/inlinfosuppobj (stderr)
memcheck/tests/leak-autofreepool-0 (stderr)
memcheck/tests/leak-autofreepool-1 (stderr)
memcheck/tests/leak-autofreepool-2 (stderr)
memcheck/tests/leak-autofreepool-4 (stderr)
-memcheck/tests/leak-autofreepool-5 (stderr)
memcheck/tests/leak-autofreepool-6 (stderr)
memcheck/tests/leak-cases-full (stderr)
+memcheck/tests/leak-cases-possible (stderr)
memcheck/tests/leak-cases-summary (stderr)
+memcheck/tests/leak-cycle (stderr)
memcheck/tests/leak-delta (stderr)
+memcheck/tests/leak-tree (stderr)
memcheck/tests/leak_cpp_interior (stderr)
memcheck/tests/lks (stderr)
+memcheck/tests/long_namespace_xml (stderr)
memcheck/tests/manuel1 (stderr)
-memcheck/tests/memalign_test (stderr)
memcheck/tests/memcmptest (stderr)
+memcheck/tests/mempool (stderr)
+memcheck/tests/mempool2 (stderr)
memcheck/tests/mismatches (stderr)
+memcheck/tests/nanoleak2 (stderr)
+memcheck/tests/nanoleak_dynsupp (stderr)
+memcheck/tests/nanoleak_supp (stderr)
memcheck/tests/origin1-yes (stderr)
memcheck/tests/origin2-not-quite (stderr)
memcheck/tests/origin3-no (stderr)
-memcheck/tests/origin4-many (stderr)
memcheck/tests/origin5-bz2 (stderr)
memcheck/tests/origin6-fp (stderr)
memcheck/tests/overlap (stderr)
+memcheck/tests/pointer-trace (stderr)
memcheck/tests/post-syscall (stderr)
+memcheck/tests/recursive-merge (stderr)
memcheck/tests/sem (stderr)
memcheck/tests/sized_delete (stderr)
memcheck/tests/static_malloc (stderr)
memcheck/tests/strchr (stderr)
@@ -60,15 +68,17 @@
memcheck/tests/supponlyobj (stderr)
memcheck/tests/suppvarinfo5 (stderr)
memcheck/tests/test-plo-no (stderr)
memcheck/tests/thread_alloca (stderr)
+memcheck/tests/threadname_xml (stderr)
+memcheck/tests/trivialleak (stderr)
+memcheck/tests/undef_malloc_args (stderr)
memcheck/tests/varinfo1 (stderr)
memcheck/tests/varinfo2 (stderr)
memcheck/tests/varinfo3 (stderr)
memcheck/tests/varinfo4 (stderr)
memcheck/tests/varinfo5 (stderr)
memcheck/tests/varinfo6 (stderr)
-memcheck/tests/wrap6 (stdout)
memcheck/tests/wrapmalloc (stdout)
memcheck/tests/wrapmallocstatic (stdout)
memcheck/tests/xml1 (stderr)
helgrind/tests/annotate_hbefore (stderr)
@@ -141,8 +151,9 @@
drd/tests/atomic_var (stderr)
drd/tests/bug-235681 (stderr)
drd/tests/circular_buffer (stderr)
drd/tests/concurrent_close (stderr)
+drd/tests/condvar (stderr)
drd/tests/custom_alloc (stderr)
drd/tests/custom_alloc_fiw (stderr)
drd/tests/dlopen (stdout)
drd/tests/dlopen (stderr)
@@ -189,10 +200,15 @@
drd/tests/sem_open (stderr)
drd/tests/sem_open2 (stderr)
drd/tests/sem_open3 (stderr)
drd/tests/sem_open_traced (stderr)
+drd/tests/shared_timed_mutex (stderr)
drd/tests/sigalrm (stderr)
drd/tests/sigaltstack (stderr)
+drd/tests/std_atomic (stderr)
+drd/tests/std_list (stderr)
+drd/tests/std_mutex (stderr)
+drd/tests/std_string (stderr)
drd/tests/str_tester (stderr)
drd/tests/swapcontext (stderr)
drd/tests/tc01_simple_race (stderr)
drd/tests/tc02_simple_tls (stderr)
@@ -219,8 +235,9 @@
drd/tests/tc22_exit_w_lock (stderr)
drd/tests/tc23_bogus_condwait (stderr)
drd/tests/threaded-fork-vcs (stderr)
drd/tests/threaded-fork (stderr)
+drd/tests/timed_mutex (stderr)
drd/tests/tls_threads (stderr)
drd/tests/trylock (stderr)
drd/tests/unit_bitmap (stderr)
drd/tests/unit_vc (stderr)
@@ -273,9 +290,8 @@
dhat/tests/single (stderr)
none/tests/allexec32 (stdout)
none/tests/allexec64 (stdout)
none/tests/amd64/amd64locked (stdout)
-none/tests/amd64/sse4-64 (stdout)
none/tests/async-sigs (stderr)
none/tests/bug234814 (stdout)
none/tests/bug234814 (stderr)
none/tests/coolo_sigaction (stdout)
@@ -294,11 +310,8 @@
none/tests/ioctl_moans (stderr)
none/tests/mmap_fcntl_bug (stderr)
none/tests/nocwd (stdout)
none/tests/nocwd (stderr)
-none/tests/pth_2sig (stderr)
-none/tests/pth_cancel1 (stderr)
-none/tests/pth_cancel2 (stderr)
none/tests/require-text-symbol-2 (stderr)
none/tests/rlimit_nofile (stderr)
none/tests/scripts/shell (stderr)
none/tests/syscall-restart1 (stderr)

Smart Diff (ubuntu-20.04): Failure

Expected tolerance 'same' but got 'mixed-worse' instead

===================================================================
--- .github/ubuntu-20.04-expected.txt
+++ results-tests.txt
@@ -1,5 +1,7 @@
memcheck/tests/overlap (stderr)
+drd/tests/pth_mutex_signal (stderr)
+drd/tests/shared_timed_mutex (stderr)
none/tests/fdleak_cmsg (stderr)
none/tests/fdleak_creat (stderr)
none/tests/fdleak_dup (stderr)
none/tests/fdleak_dup2 (stderr)
@@ -9,5 +11,4 @@
none/tests/fdleak_pipe (stderr)
none/tests/fdleak_socketpair (stderr)
none/tests/rlimit64_nofile (stderr)
none/tests/rlimit_nofile (stderr)
-none/tests/scripts/shell (stderr)

Hi @madsmtm,

Sorry for not getting back to you before but could you explain what difference moving the loop does?

I can't remember honestly